﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace DirectoryEnumeratorAsync
{
    class Program
    {
        private static log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
        private static readonly TimeSpan taskSchedulerIdleTimeout = TimeSpan.FromSeconds(15);

        static void Main(string[] args)
        {
            if (args.Length == 1)
            {
                PerformGetFileSystemEntriesTest(args[0]);
            }
            else
            {
                Console.WriteLine("Usage: DirectoryEnumeratorAsync <path>");
            }
        }

        public static async void PerformGetFileSystemEntriesTest(string pathToEnumerate)
        {
            Console.WriteLine("{0} {1}", DateTime.Now.TimeOfDay.ToString(), Extensions.CurrentMethodName());
            Console.WriteLine("Getting directories and files for path: {0}", pathToEnumerate);

            using (new ProcessPrivileges.PrivilegeEnabler(System.Diagnostics.Process.GetCurrentProcess(),
                ProcessPrivileges.Privilege.Backup, ProcessPrivileges.Privilege.SystemEnvironment))
            {
                if (!Directory.Exists(pathToEnumerate))
                {
                    Console.WriteLine("Directory does not exist: {0}", pathToEnumerate);
                    return;
                }

                // Performance test: Server: HP DL380 G5 2x dual-core CPU; Target: NetApp SMB Storage; 1 Gbps network connection
                // Time required: 00:09:17; Processor time: 00:19:43
                // Directories: 358701; Reparse Points: 0 Files: 392463
                // Note: 20x faster than the equivalent synchronous test
                Console.WriteLine("Asynchronous test...");
                var task = TestGetFileSystemEntriesAsync(pathToEnumerate);
                task.Wait();
                Console.WriteLine();

                // Time required: 03:07:25; Processor time: 00:11:24
                // Directories: 358701; Reparse Points: 0 Files: 392463
                Console.WriteLine("Synchronous test...");
                TestGetFileSystemEntries(pathToEnumerate);
                Console.WriteLine();
            }

            Console.WriteLine("Finished.");
        }

        public static ConcurrentDictionary<string, FileInfo> TestGetFileSystemEntries(string pathToEnumerate)
        {
            log.InfoFormat("[ThreadId: {0}] {1}", Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName());

            DateTime startTime = DateTime.UtcNow;
            TimeSpan startProcessorTime = Process.GetCurrentProcess().TotalProcessorTime;

            ConcurrentDictionary<string, FileInfo> fileSystemEntries = null;
            ConcurrentBag<string> exclusions = new ConcurrentBag<string>();
            exclusions.Add("$Recycle.Bin");

            try
            {
                fileSystemEntries = GetFileSystemEntries(pathToEnumerate, exclusions, null);
            }
            catch (Exception e)
            {
                log.ErrorFormat("[ThreadId: {0}] {1} {2}",
                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(), e.VerboseExceptionString());
            }
            finally
            {
                TimeSpan elapsedTime = DateTime.UtcNow.Subtract(startTime);
                TimeSpan processorTime = Process.GetCurrentProcess().TotalProcessorTime.Subtract(startProcessorTime);

                log.InfoFormat("[ThreadId: {0}] {1} Finished.  Time required: {2}; Processor time: {3}",
                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                    String.Format("{0:hh}:{0:mm}:{0:ss}", elapsedTime), String.Format("{0:hh}:{0:mm}:{0:ss}", processorTime));

                int directoryCount = fileSystemEntries.Where(x => (x.Value.Attributes & FileAttributes.Directory) == FileAttributes.Directory).Count();
                int reparsePointCount = fileSystemEntries.Where(x => (x.Value.Attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint).Count();
                int fileCount = fileSystemEntries.Count - directoryCount;

                log.InfoFormat("[ThreadId: {0}] {1} Directories: {2}; Reparse Points: {3} Files: {4}",
                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                    directoryCount, reparsePointCount, fileCount);
            }

            return fileSystemEntries;
        }
        private static ConcurrentDictionary<string, FileInfo> GetFileSystemEntries(
            string path, ConcurrentBag<string> exclusions, ConcurrentDictionary<string, FileInfo> fileSystemEntries)
        {
            return GetFileSystemEntries(new FileInfo(path), exclusions, fileSystemEntries);
        }
        private static ConcurrentDictionary<string, FileInfo> GetFileSystemEntries(
            FileInfo pathFileInfo, ConcurrentBag<string> exclusions, ConcurrentDictionary<string, FileInfo> fileSystemEntries)
        {
            if (fileSystemEntries == null) fileSystemEntries = new ConcurrentDictionary<string, FileInfo>(StringComparer.CurrentCultureIgnoreCase);
            if (fileSystemEntries.Count % 5000 == 0)
            {
                log.DebugFormat("[ThreadId: {0}] {1} Item count: {2}; Current Path: {3}",
                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                    fileSystemEntries.Count, pathFileInfo.FullName);
            }

            if ((pathFileInfo.Attributes & FileAttributes.Directory) != FileAttributes.Directory)
            {
                throw new ArgumentException(String.Format("Path is not a directory: {0}", pathFileInfo.FullName));
            }

            foreach (var fileSystemEntryPath in Directory.EnumerateFileSystemEntries(pathFileInfo.FullName, "*", SearchOption.TopDirectoryOnly))
            {
                FileInfo childFileInfo = null;

                try
                {
                    childFileInfo = new FileInfo(fileSystemEntryPath);
                    FileInfo placeHolder = null;
                    fileSystemEntries.AddOrUpdate(childFileInfo.FullName, childFileInfo, (TKey, TOldValue) => placeHolder);

                    if (((childFileInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) && ((pathFileInfo.Attributes & FileAttributes.ReparsePoint) != FileAttributes.ReparsePoint))
                    {
                        if (!exclusions.Any(x => childFileInfo.FullName.Contains(x)))
                        {
                            GetFileSystemEntries(childFileInfo, exclusions, fileSystemEntries);
                        }
                    }
                }
                catch (UnauthorizedAccessException)
                {
                    log.WarnFormat("[ThreadId: {0}] {1} Insufficient permissions to access path: {2}",
                        Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(), fileSystemEntryPath);
                }
                catch (PathTooLongException)
                {
                    log.WarnFormat("[ThreadId: {0}] {1} path too long: {2}",
                        Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(), fileSystemEntryPath);
                }
                catch (Exception e)
                {
                    log.WarnFormat("[ThreadId: {0}] {1} path: {2} child path: {3} Exception: {4}",
                        Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                        pathFileInfo.FullName, fileSystemEntryPath, e.VerboseExceptionString());
                    throw;
                }
            }

            return fileSystemEntries;
        }

        public static async Task<ConcurrentDictionary<string, FileInfo>> TestGetFileSystemEntriesAsync(string pathToEnumerate)
        {
            log.InfoFormat("[ThreadId: {0}] {1}", Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName());

            DateTime startTime = DateTime.UtcNow;
            TimeSpan startProcessorTime = Process.GetCurrentProcess().TotalProcessorTime;

            ConcurrentDictionary<string, FileInfo> fileSystemEntries = null;
            ConcurrentBag<string> exclusions = new ConcurrentBag<string>();
            exclusions.Add("$Recycle.Bin");
            ConcurrentBag<Task> tasks = new ConcurrentBag<Task>();

            Progress<Tuple<long, string>> progress = new Progress<Tuple<long, string>>((reportedProgress) =>
            {
                log.DebugFormat("[ThreadId: {0}] {1} Item count: {2}; Current Path: {3}",
                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                    reportedProgress.Item1.ToString(), reportedProgress.Item2);
            });

            await Task.Run(async () =>
                {
                    try
                    {
                        var getFileSystemEntriesTask = GetFileSystemEntriesAsync(new FileInfo(pathToEnumerate), tasks, exclusions, null, progress);
                        tasks.Add(getFileSystemEntriesTask);

                        int taskCount = tasks.Count;
                        while (true)
                        {
                            await Task.Delay(taskSchedulerIdleTimeout);
                            Task.WaitAll(tasks.ToArray());

                            if ((tasks.Count == taskCount) && !tasks.Any(x => !x.IsCompleted)) break;
                            log.InfoFormat("[ThreadId: {0}] {1} Total tasks: {2}; Waiting for: {3} remaining tasks to complete.",
                                Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                                tasks.Count, tasks.Where(x => !x.IsCompleted).Count());                            
                            taskCount = tasks.Count;
                        }

                        fileSystemEntries = getFileSystemEntriesTask.Result;
                    }
                    catch (Exception e)
                    {
                        log.ErrorFormat("[ThreadId: {0}] {1} Path: {2} Exception: {3}",
                            Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(), pathToEnumerate, e.VerboseExceptionString());
                        throw;
                    }
                    finally
                    {
                        TimeSpan elapsedTime = DateTime.UtcNow.Subtract(startTime);
                        TimeSpan processorTime = Process.GetCurrentProcess().TotalProcessorTime.Subtract(startProcessorTime);

                        log.InfoFormat("[ThreadId: {0}] {1} Finished.  Time required: {2}; Processor time: {3}",
                            Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                            String.Format("{0:hh}:{0:mm}:{0:ss}", elapsedTime), String.Format("{0:hh}:{0:mm}:{0:ss}", processorTime));

                        int directoryCount = fileSystemEntries.Where(x => (x.Value.Attributes & FileAttributes.Directory) == FileAttributes.Directory).Count();
                        int reparsePointCount = fileSystemEntries.Where(x => (x.Value.Attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint).Count();
                        int fileCount = fileSystemEntries.Count - directoryCount;

                        log.InfoFormat("[ThreadId: {0}] {1} Directories: {2}; Reparse Points: {3} Files: {4}",
                            Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                            directoryCount, reparsePointCount, fileCount);
                    }
                });

            return fileSystemEntries;
        }
        private static async Task<ConcurrentDictionary<string, FileInfo>> GetFileSystemEntriesAsync(
            FileInfo pathFileInfo, 
            ConcurrentBag<Task> tasks, ConcurrentBag<string> exclusions,
            ConcurrentDictionary<string, FileInfo> fileSystemEntries,
            IProgress<Tuple<long, string>> progress)
        {
            if (fileSystemEntries == null) fileSystemEntries = new ConcurrentDictionary<string, FileInfo>(StringComparer.CurrentCultureIgnoreCase);

            if ((progress != null) && (fileSystemEntries.Count % 5000 == 0))
            {
                progress.Report(new Tuple<long, string>(fileSystemEntries.LongCount(), pathFileInfo.FullName));
            }

            try
            {
                await Task.Run(() =>
                    {
                        if ((pathFileInfo.Attributes & FileAttributes.Directory) != FileAttributes.Directory)
                        {
                            throw new ArgumentException(String.Format("Path is not a directory: {0}", pathFileInfo.FullName));
                        }

                        foreach (var fileSystemEntryPath in Directory.EnumerateFileSystemEntries(pathFileInfo.FullName, "*", SearchOption.TopDirectoryOnly))
                        {
                            FileInfo childFileInfo = null;

                            try
                            {
                                childFileInfo = new FileInfo(fileSystemEntryPath);
                                FileInfo placeHolder = null;
                                fileSystemEntries.AddOrUpdate(childFileInfo.FullName, childFileInfo, (TKey, TOldValue) => placeHolder);

                                if (((childFileInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) && ((pathFileInfo.Attributes & FileAttributes.ReparsePoint) != FileAttributes.ReparsePoint))
                                {
                                    if (!exclusions.Any(x => childFileInfo.FullName.Contains(x)))
                                    {
                                        tasks.Add(
                                            Task.Factory.StartNew(async (x) =>
                                            {
                                                await GetFileSystemEntriesAsync(childFileInfo, tasks, exclusions, fileSystemEntries, progress);
                                            }, childFileInfo.FullName)
                                        );
                                    }
                                }
                            }
                            catch (UnauthorizedAccessException)
                            {
                                log.WarnFormat("[ThreadId: {0}] {1} Insufficient permissions to access path: {2}",
                                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(), fileSystemEntryPath);
                            }
                            catch (PathTooLongException)
                            {
                                log.WarnFormat("[ThreadId: {0}] {1} path too long: {2}",
                                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(), fileSystemEntryPath);
                            }
                            catch (Exception e)
                            {
                                log.WarnFormat("[ThreadId: {0}] {1} path: {2} child path: {3} Exception: {4}",
                                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(),
                                    pathFileInfo.FullName, fileSystemEntryPath, e.VerboseExceptionString());
                                throw;
                            }
                        }
                    });
            }
            catch (Exception e)
            {
                log.WarnFormat("[ThreadId: {0}] {1} path: {2} Exception: {3}",
                    Thread.CurrentThread.ManagedThreadId.ToString(), Extensions.CurrentMethodName(), pathFileInfo.FullName, e.VerboseExceptionString());
                throw;
            }

            return fileSystemEntries;
        }
    }
}
